# -*- coding: utf-8 -*-

import scrapy
import re
from test1.items import miitnongyao
import datetime

class miitspider(scrapy.Spider):
    name = 'nongyao'
    start_urls = ['http://data.miit.gov.cn/datainfo/miit/nycpscpzzsmd.jsp']
    count = 1
    def parse(self, response):
        page_total = response.xpath("//div[@id='divContent']/table[2]/tr/td[1]/div/table/tr/td[1]/font[3]/text()").extract_first()
        page_total_int = int(page_total)
        #获取总页数
        for page in range(1,page_total_int+1):
            yield scrapy.FormRequest('http://data.miit.gov.cn/datainfo/miit/nycpscpzzsmd.jsp',
                                     formdata={
                                            'count':'18520',
                                            'pages':str(page_total),
                                            'page':str(page)
                                     },
                                     callback=self.parse_info
                                     )

    def parse_info(self,response):
        nongyao_info = miitnongyao()
        tr_list = response.xpath("//div[@id='divContent']/table[2]/tr/td[1]/table[4]/tr/td[2]/table[@class='list-table']/tr")
        for tr in tr_list:
            if tr.xpath("td[1]/text()").extract_first():
                nongyao_info['serial_number'] = tr.xpath("td[1]/text()").extract_first().strip()
                nongyao_info['province'] = tr.xpath("td[2]/text()").extract_first().strip()
                nongyao_info['the_name_of_the_enterprise'] = tr.xpath("td[3]/text()").extract_first().strip()
                nongyao_info['product_categories'] = tr.xpath("td[4]/text()").extract_first().strip()
                nongyao_info['product_name'] = tr.xpath("td[5]/text()").extract_first().strip()
                nongyao_info['production_type'] = tr.xpath("td[6]/text()").extract_first().strip()
                nongyao_info['the_implementation_of_standards'] = tr.xpath("td[7]/text()").extract_first().strip()
                nongyao_info['production_approval_number'] = tr.xpath("td[8]/text()").extract_first().strip()
                nongyao_info['the_date_of_issuance'] = tr.xpath("td[9]/text()").extract_first().strip()
                nongyao_info['valid'] = tr.xpath("td[10]/text()").extract_first().strip()
                nongyao_info['insert_time'] = str(datetime.datetime.now())
                nongyao_info['url'] = response.url
                yield nongyao_info